package com.luoqifei.client;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.sql.*;
import org.apache.spark.sql.types.DataTypes;
import org.apache.spark.sql.types.Metadata;
import org.apache.spark.sql.types.StructField;
import org.apache.spark.sql.types.StructType;

import java.util.ArrayList;
import java.util.List;

/**
 * Created by luoqifei on 17-2-28.
 */
public class GenDataFrame {
    public static void test(){
        Dataset<Row> dataSet = null;
        SparkConf sConf = new SparkConf().setMaster("local").setAppName("Test Kmx InputFormat");

        SparkSession sparkSession = SparkSession.builder().config(sConf).getOrCreate();
        List<String> cols = new ArrayList<>();
        cols.add("a");
        cols.add("b");
        cols.add("a1");
        List<StructField> fields = new ArrayList<>();
        for(String col : cols){
            fields.add(DataTypes.createStructField(col,DataTypes.StringType,true, Metadata.empty()));
        }
        StructType schema = DataTypes.createStructType(fields);
//        StructType schema = new StructType(new StructField[]{
//                new StructField("a", DataTypes.StringType,false, Metadata.empty()),
//                new StructField("b", DataTypes.StringType,false, Metadata.empty())
//                //new StructField("c", DataTypes.StringType,false, Metadata.empty())
//        });
        SQLContext sqlContext = sparkSession.sqlContext();
        JavaRDD<Row> test = sparkSession.read().textFile("./java-spark/src/main/resources/csv.txt").javaRDD().map(new Function<String, Row>() {
            @Override
            public Row call(String s) throws Exception {
                return RowFactory.create(s.split(","));
            }
        });
        dataSet = sqlContext.createDataFrame(test,schema);
        dataSet.show();
        dataSet.printSchema();
        sparkSession.stop();
    }
    public static void main(String args[]){
        GenDataFrame.test();
    }
}
